from transformers import AutoTokenizer, AutoModelForTokenClassification
from transformers import pipeline

# tokenizer = AutoTokenizer.from_pretrained("dslim/distilbert-NER")
# model = AutoModelForTokenClassification.from_pretrained("dslim/distilbert-NER")
#
# nlp = pipeline("ner", model=model, tokenizer=tokenizer)
# # example = "My name is Wolfgang and I live in Berlin"
# example = "在公交站，突然恶心，已经持续刚才，很难受"
#
# ner_results = nlp(example)
# print(ner_results)

from transformers import AutoModel, AutoTokenizer

# 模型名称，例如 "shibing624/text2vec-base-chinese"
model_name = "distilbert-base-multilingual-cased"

# 下载并加载模型和分词器
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModel.from_pretrained(model_name)

# 保存到自定义路径（可选）
model.save_pretrained("./models")
tokenizer.save_pretrained("./models")
